SherbyNet - /page/wikibook/entry.mu

Micron Document

<!DOCTYPE html>

</head>

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .infobox-subbox{padding:0;border:none;margin:-3px;width:auto;min-width:100%;font-size:100%;clear:none;float:none;background-color:transparent}.mw-parser-output .infobox-3cols-child{margin:auto}.mw-parser-output .infobox .navbar{font-size:100%}@media screen{html.skin-theme-clientpref-night .mw-parser-output .infobox-full-data:not(.notheme)>div:not(.notheme)[style]{background:#1f1f23!important;color:#f8f9fa}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .infobox-full-data:not(.notheme)>div:not(.notheme)[style]{background:#1f1f23!important;color:#f8f9fa}}@media(min-width:640px){body.skin--responsive .mw-parser-output .infobox-table{display:table!important}body.skin--responsive .mw-parser-output .infobox-table>caption{display:table-caption!important}body.skin--responsive .mw-parser-output .infobox-table>tbody{display:table-row-group}body.skin--responsive .mw-parser-output .infobox-table th,body.skin--responsive .mw-parser-output .infobox-table td{padding-left:inherit;padding-right:inherit}}

/* end https://en.wikipedia.org/ */

</style><table class="infobox vevent"><tbody><tr><th colspan="2" class="infobox-above summary">word2vec</th></tr><tr><th scope="row" class="infobox-label" style="white-space: nowrap;"><a href="Programmer" title="Programmer">Original author(s)</a></th><td class="infobox-data"><a href="Google_AI" title="Google AI">Google AI</a></td></tr><tr><th scope="row" class="infobox-label" style="white-space: nowrap;">Initial release</th><td class="infobox-data">July 29, 2013.<span style="display:none"> (<span class="bday dtstart published updated">July 29, 2013.</span>)</span></td></tr><tr><th scope="row" class="infobox-label" style="white-space: nowrap;"><a href="Repository_(version_control)" title="Repository (version control)">Repository</a></th><td class="infobox-data"><a rel="nofollow" class="external free" href="https://code.google.com/archive/p/word2vec/">https://code.google.com/archive/p/word2vec/</a></td></tr><tr><th scope="row" class="infobox-label" style="white-space: nowrap;"><a href="Software_categories#Categorization_approaches" title="Software categories">Type</a></th><td class="infobox-data"><style data-mw-deduplicate="TemplateStyles:r1126788409">

/* start https://en.wikipedia.org/ */

.mw-parser-output .plainlist ol,.mw-parser-output .plainlist ul{line-height:inherit;list-style:none;margin:0;padding:0}.mw-parser-output .plainlist ol li,.mw-parser-output .plainlist ul li{margin-bottom:0}

/* end https://en.wikipedia.org/ */

</style><div class="plainlist" style="margin-left:1em;text-indent:-1em;">

<ul><li><a href="Language_model" title="Language model">Language model</a></li>

<li><a href="Word_embedding" title="Word embedding">Word embedding</a></li></ul>

</div></td></tr><tr><th scope="row" class="infobox-label" style="white-space: nowrap;"><a href="Software_license" title="Software license">License</a></th><td class="infobox-data"><a href="Apache-2.0" class="mw-redirect" title="Apache-2.0">Apache-2.0</a></td></tr></tbody></table><style data-mw-deduplicate="TemplateStyles:r1129693374">

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r886047488">

/* start https://en.wikipedia.org/ */

.mw-parser-output .nobold{font-weight:normal}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Machine_learning" title="Machine learning">Machine learning</a><br>and <a href="Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content">

<ul><li><a href="Supervised_learning" title="Supervised learning">Supervised learning</a></li>

<li><a href="Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li>

<li><a href="Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li>

<li><a href="Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li>

<li><a href="Online_machine_learning" title="Online machine learning">Online learning</a></li>

<li><a href="Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li>

<li><a href="Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li>

<li><a href="Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li>

<li><a href="Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li>

<li><a href="Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li>

<li><a href="Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Statistical_classification" title="Statistical classification">Classification</a></li>

<li><a href="Generative_model" title="Generative model">Generative modeling</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li>

<li><a href="Density_estimation" title="Density estimation">Density estimation</a></li>

<li><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li>

<li><a href="Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li>

<li><a href="Automated_machine_learning" title="Automated machine learning">AutoML</a></li>

<li><a href="Association_rule_learning" title="Association rule learning">Association rules</a></li>

<li><a href="Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li>

<li><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></li>

<li><a href="Feature_engineering" title="Feature engineering">Feature engineering</a></li>

<li><a href="Feature_learning" title="Feature learning">Feature learning</a></li>

<li><a href="Learning_to_rank" title="Learning to rank">Learning to rank</a></li>

<li><a href="Grammar_induction" title="Grammar induction">Grammar induction</a></li>

<li><a href="Ontology_learning" title="Ontology learning">Ontology learning</a></li>

<li><a href="Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="Supervised_learning" title="Supervised learning">Supervised learning</a><br><span class="nobold"><span style="font-size: 85%;">(<b><a href="Statistical_classification" title="Statistical classification">classification</a></b> • <b><a href="Regression_analysis" title="Regression analysis">regression</a></b>)</span></span> </div></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li>

<li><a href="Decision_tree_learning" title="Decision tree learning">Decision trees</a></li>

<li><a href="Ensemble_learning" title="Ensemble learning">Ensembles</a>

<ul><li><a href="Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li>

<li><a href="Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li>

<li><a href="Random_forest" title="Random forest">Random forest</a></li></ul></li>

<li><a href="Linear_regression" title="Linear regression">Linear regression</a></li>

<li><a href="Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li>

<li><a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li>

<li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Perceptron" title="Perceptron">Perceptron</a></li>

<li><a href="Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li>

<li><a href="Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="BIRCH" title="BIRCH">BIRCH</a></li>

<li><a href="Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li>

<li><a href="K-means_clustering" title="K-means clustering"><i>k</i>-means</a></li>

<li><a href="Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li>

<li><a href="Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li>

<li><br><a href="DBSCAN" title="DBSCAN">DBSCAN</a></li>

<li><a href="OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li>

<li><a href="Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Factor_analysis" title="Factor analysis">Factor analysis</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Graphical_model" title="Graphical model">Graphical models</a>

<ul><li><a href="Bayesian_network" title="Bayesian network">Bayes net</a></li>

<li><a href="Conditional_random_field" title="Conditional random field">Conditional random field</a></li>

<li><a href="Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Random_sample_consensus" title="Random sample consensus">RANSAC</a></li>

<li><a href="Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li>

<li><a href="Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Neural networks</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Deep_learning" title="Deep learning">Deep learning</a></li>

<li><a href="Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network</a>

<li><a href="Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li>

<li><a href="Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a>

<ul><li><a href="Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li>

<li><a href="Diffusion_model" title="Diffusion model">Diffusion model</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a>

<li><a href="LeNet" title="LeNet">LeNet</a></li>

<li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="DeepDream" title="DeepDream">DeepDream</a></li></ul></li>

<li><a href="Neural_field" title="Neural field">Neural field</a>

<ul><li><a href="Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li>

<li><a href="Physics-informed_neural_networks" title="Physics-informed neural networks">Physics-informed neural networks</a></li></ul></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision</a></li></ul></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li>

<li><a href="Memtransistor" title="Memtransistor">Memtransistor</a></li>

<li><a href="Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li>

<li><a href="Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li>

<li><a href="Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li>

<li><a href="Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li>

<li><a href="Mechanistic_interpretability" title="Mechanistic interpretability">Mechanistic interpretability</a></li>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li>

<li><a href="Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li>

<li><a href="Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li>

<li><a href="Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li>

<li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li>

<li><a href="Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li>

<li><a href="Occam_learning" title="Occam learning">Occam learning</a></li>

<li><a href="Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li>

<li><a href="Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li>

<li><a href="Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li>

<li><a href="Topological_deep_learning" title="Topological deep learning">Topological deep learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist">

<li><a href="Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li>

<li><a href="International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li>

<li><a href="List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a>

<ul><li><a href="List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li>

<li><a href="Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

<p><b>Word2vec</b> is a technique in <a href="Natural_language_processing" title="Natural language processing">natural language processing</a> (NLP) for obtaining <a href="Vector_space" title="Vector space">vector</a> representations of words. These vectors capture information about the meaning of the word based on the surrounding words. The word2vec algorithm estimates these representations by modeling text in a large <a href="Corpus_of_text" class="mw-redirect" title="Corpus of text">corpus</a>. Once trained, such a model can detect <a href="Synonym" title="Synonym">synonymous</a> words or suggest additional words for a partial sentence. Word2vec was developed by <a href="Tom%C3%A1%C5%A1_Mikolov" title="Tomáš Mikolov">Tomáš Mikolov</a>, Kai Chen, Greg Corrado, <a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a> and <a href="Jeff_Dean" title="Jeff Dean">Jeff Dean</a> at Google, and published in 2013.<sup id="cite_ref-mikolov_1-0" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-mikolov-nips_2-0" class="reference"><a href="#cite_note-mikolov-nips-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup>

</p><p>Word2vec represents a word as a high-dimension <a href="Vector_(geometry)" class="mw-redirect" title="Vector (geometry)">vector</a> of numbers which capture relationships between words. In particular, words which appear in similar contexts are mapped to vectors which are nearby as measured by <a href="Cosine_similarity" title="Cosine similarity">cosine similarity</a>. This indicates the level of <a href="Semantic_similarity" title="Semantic similarity">semantic similarity</a> between the words, so for example the vectors for <i>walk</i> and <i>ran</i> are nearby, as are those for "but" and "however", and "Berlin" and "Germany".

</p>

<div class="mw-heading mw-heading2"><h2 id="Approach">Approach</h2></div>

<p>Word2vec is a group of related models that are used to produce <a href="Word_embedding" title="Word embedding">word embeddings</a>. These models are shallow, two-layer <a href="Neural_network" title="Neural network">neural networks</a> that are trained to reconstruct linguistic contexts of words. Word2vec takes as its input a large <a href="Text_corpus" title="Text corpus">corpus of text</a> and produces a mapping of the set of words to a <a href="Vector_space" title="Vector space">vector space</a>, typically of several hundred <a href="Dimensions" class="mw-redirect" title="Dimensions">dimensions</a>, with each unique word in the <a href="Corpus_linguistics" title="Corpus linguistics">corpus</a> being assigned a vector in the space.

</p><p>Word2vec can use either of two model architectures to produce these <a href="Distributed_representation" class="mw-redirect" title="Distributed representation">distributed representations</a> of words: continuous <a href="Bag-of-words" class="mw-redirect" title="Bag-of-words">bag of words</a> (CBOW) or continuously sliding skip-gram. In both architectures, word2vec considers both individual words and a sliding context window as it iterates over the corpus.

</p><p>The CBOW can be viewed as a ‘fill in the blank’ task, where the word embedding represents the way the word influences the relative probabilities of other words in the context window. Words which are semantically similar should influence these probabilities in similar ways, because semantically similar words should be used in similar contexts. The order of context words does not influence prediction (bag of words assumption).

</p><p>In the continuous skip-gram architecture, the model uses the current word to predict the surrounding window of context words.<sup id="cite_ref-mikolov_1-1" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-mikolov-nips_2-1" class="reference"><a href="#cite_note-mikolov-nips-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> The skip-gram architecture weighs nearby context words more heavily than more distant context words. According to the authors' note,<sup id="cite_ref-:1_3-0" class="reference"><a href="#cite_note-:1-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> CBOW is faster while skip-gram does a better job for infrequent words.

</p><p>After the model is trained, the learned word embeddings are positioned in the vector space such that words that share common contexts in the corpus — that is, words that are <a href="Semantic_similarity" title="Semantic similarity">semantically</a> and syntactically similar — are located close to one another in the space.<sup id="cite_ref-mikolov_1-2" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> More dissimilar words are located farther from one another in the space.<sup id="cite_ref-mikolov_1-3" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Mathematical_details">Mathematical details</h2></div>

<p>This section is based on expositions.<sup id="cite_ref-explain_4-0" class="reference"><a href="#cite_note-explain-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-5" class="reference"><a href="#cite_note-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup>

</p><p>A corpus is a sequence of words. Both CBOW and skip-gram are methods to learn one vector per word appearing in the corpus.

</p><p>Let <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span> ("vocabulary") be the set of all words appearing in the corpus <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle C}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle C}</annotation>

</semantics>

</math></span><img src="./4fc55753007cd3c18576f7933f6f089196732029.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.766ex; height:2.176ex;" alt="{\displaystyle C}" loading="lazy"></span>. Our goal is to learn one vector <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle v_{w}\in \mathbb {R} ^{n}}">

<msub>

</mrow>

</msub>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle v_{w}\in \mathbb {R} ^{n}}</annotation>

</semantics>

</math></span><img src="./10ffad67d629d0df21f0aac849e8f7ebf8ed370e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.274ex; height:2.676ex;" alt="{\displaystyle v_{w}\in \mathbb {R} ^{n}}" loading="lazy"></span> for each word <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w\in V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w\in V}</annotation>

</semantics>

</math></span><img src="./fb3ba2e494febb4b85886a94ea45400bbfa30176.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.292ex; height:2.176ex;" alt="{\displaystyle w\in V}" loading="lazy"></span>.

</p><p>The idea of skip-gram is that the vector of a word should be close to the vector of each of its neighbors. The idea of CBOW is that the vector-sum of a word's neighbors should be close to the vector of the word.

</p>

<div class="mw-heading mw-heading3"><h3 id="Continuous_bag-of-words_(CBOW)">Continuous bag-of-words (CBOW)</h3></div>

<p>The idea of CBOW is to represent each word with a vector, such that it is possible to predict a word using the sum of the vectors of its neighbors. Specifically, for each word <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation>

</semantics>

</math></span><img src="./fe22f0329d3ecb2e1880d44d191aba0e5475db68.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" loading="lazy"></span> in the corpus, the one-hot encoding of the word is used as the input to the neural network. The output of the neural network is a probability distribution over the dictionary, representing a prediction of individual words in the neighborhood of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation>

</semantics>

</math></span><img src="./fe22f0329d3ecb2e1880d44d191aba0e5475db68.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" loading="lazy"></span>. The objective of training is to maximize <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sum _{i}\ln \Pr(w_{i}|w_{j}:j\in i+N)}">

</mrow>

</munder>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sum _{i}\ln \Pr(w_{i}|w_{j}:j\in i+N)}</annotation>

</semantics>

</math></span><img src="./d5126351a8d1f89effe1f8c74eaf5059dc2bfb82.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:27.499ex; height:5.509ex;" alt="{\displaystyle \sum _{i}\ln \Pr(w_{i}|w_{j}:j\in i+N)}" loading="lazy"></span>.

</p><p>For example, if we want each word in the corpus to be predicted by every other word in a small span of 4 words. The set of relative indexes of neighbor words will be: <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N=\{-2,-1,+1,+2\}}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle N=\{-2,-1,+1,+2\}}</annotation>

</semantics>

</math></span><img src="./82b604882862f765a9adf0a11a525ed9eff47634.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.471ex; height:2.843ex;" alt="{\displaystyle N=\{-2,-1,+1,+2\}}" loading="lazy"></span>, and the objective is <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sum _{i}\ln \Pr(w_{i}|w_{i-2},w_{i-1},w_{i+1},w_{i+2})}">

</mrow>

</munder>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sum _{i}\ln \Pr(w_{i}|w_{i-2},w_{i-1},w_{i+1},w_{i+2})}</annotation>

</semantics>

</math></span><img src="./6d9614e3d32414de557bd056269bb3e569197546.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:34.842ex; height:5.509ex;" alt="{\displaystyle \sum _{i}\ln \Pr(w_{i}|w_{i-2},w_{i-1},w_{i+1},w_{i+2})}" loading="lazy"></span>.

</p><p>In standard bag-of-words, a word's context is represented by a word-count (aka a word <a href="Histogram" title="Histogram">histogram</a>) of its neighboring words. For example, the "sat" in "the cat sat on the mat" is represented as {"the": 2, "cat": 1, "on": 1}. Note that the last word "mat" is not used to represent "sat", because it is outside the neighborhood <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N=\{-2,-1,+1,+2\}}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle N=\{-2,-1,+1,+2\}}</annotation>

</semantics>

</p><p>In continuous bag-of-words, the histogram is multiplied by a matrix <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span> to obtain a continuous representation of the word's context. The matrix <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span> is also called a <b>dictionary</b>. Its columns are the word vectors. It has <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle D}</annotation>

</semantics>

</math></span><img src="./f34a0c600395e5d4345287e21fb26efd386990e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.924ex; height:2.176ex;" alt="{\displaystyle D}" loading="lazy"></span> columns, where <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle D}</annotation>

</semantics>

</math></span><img src="./f34a0c600395e5d4345287e21fb26efd386990e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.924ex; height:2.176ex;" alt="{\displaystyle D}" loading="lazy"></span> is the size of the dictionary. Let <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle d}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle d}</annotation>

</semantics>

</math></span><img src="./e85ff03cbe0c7341af6b982e47e9f90d235c66ab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.216ex; height:2.176ex;" alt="{\displaystyle d}" loading="lazy"></span> be the length of each word vector. We have <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V\in \mathbb {R} ^{d\times D}}">

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V\in \mathbb {R} ^{d\times D}}</annotation>

</semantics>

</math></span><img src="./61168185b0b99958520fc692686cd2533740ac30.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:10.037ex; height:2.676ex;" alt="{\displaystyle V\in \mathbb {R} ^{d\times D}}" loading="lazy"></span>.

</p><p>For example, multiplying the word histogram {"the": 2, "cat": 1, "on": 1} with <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span>, we obtain <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 2v_{\text{the}}+v_{\text{cat}}+v_{\text{on}}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 2v_{\text{the}}+v_{\text{cat}}+v_{\text{on}}}</annotation>

</semantics>

</math></span><img src="./f185d6095243b0dc3fe5e421aae98e685410c77e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:17.134ex; height:2.509ex;" alt="{\displaystyle 2v_{\text{the}}+v_{\text{cat}}+v_{\text{on}}}" loading="lazy"></span>.

</p><p>This is then multiplied with another matrix <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V'}</annotation>

</semantics>

</math></span><img src="./ff31fe992a31b4c954a933f1be91e2739a1c0ae7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.602ex; height:2.509ex;" alt="{\displaystyle V'}" loading="lazy"></span> of shape <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbb {R} ^{D\times d}}">

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbb {R} ^{D\times d}}</annotation>

</semantics>

</math></span><img src="./39147db56bc1a551a698196db4c7a329cfa10330.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.409ex; height:2.676ex;" alt="{\displaystyle \mathbb {R} ^{D\times d}}" loading="lazy"></span>. Each row of it is a word vector <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle v'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle v'}</annotation>

</semantics>

</math></span><img src="./19f1c5b0af1f30c41cb8bc0ba3bfea98681da268.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.812ex; height:2.509ex;" alt="{\displaystyle v'}" loading="lazy"></span>. This results in a vector of length <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle D}</annotation>

</semantics>

</math></span><img src="./f34a0c600395e5d4345287e21fb26efd386990e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.924ex; height:2.176ex;" alt="{\displaystyle D}" loading="lazy"></span>, one entry per dictionary entry. Then, apply the softmax to obtain a probability distribution over the dictionary.

</p><p>This system can be visualized as a neural network, similar in spirit to an <a href="Autoencoder" title="Autoencoder">autoencoder</a>, of architecture linear-linear-<a href="Softmax_function" title="Softmax function">softmax</a>, as depicted in the diagram. The system is trained by gradient descent to minimize the <a href="Cross_entropy_loss" class="mw-redirect" title="Cross entropy loss">cross-entropy loss</a>.

</p><p>In full formula, the cross-entropy loss is:<span class="mwe-math-element mwe-math-element-block"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle -\sum _{i}\ln {\frac {e^{v_{w_{i}}'\cdot (\sum _{j\in i+N}v_{w_{j}})}}{\sum _{w'}e^{v_{w'}'\cdot (\sum _{j\in i+N}v_{w_{j}})}}}}">

</mrow>

</munder>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msubsup>

</mrow>

</munder>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

<msup>

</msup>

</mrow>

</munder>

<msup>

<msup>

</msup>

</mrow>

</msubsup>

</mrow>

</munder>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle -\sum _{i}\ln {\frac {e^{v_{w_{i}}'\cdot (\sum _{j\in i+N}v_{w_{j}})}}{\sum _{w'}e^{v_{w'}'\cdot (\sum _{j\in i+N}v_{w_{j}})}}}}</annotation>

</semantics>

</math></span></span>where the outer summation <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sum _{i}}">

</mrow>

</munder>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sum _{i}}</annotation>

</semantics>

</math></span><img src="./984afb04567819f6ffc8af62378ab24c6a816398.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:3.355ex; height:5.509ex;" alt="{\displaystyle \sum _{i}}" loading="lazy"></span> is over the words in a corpus, the quantity <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sum _{j\in i+N}v_{w_{j}}}">

</mrow>

</munder>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sum _{j\in i+N}v_{w_{j}}}</annotation>

</semantics>

</math></span><img src="./d7040a413902d8c543deae6416252c270d50e877.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; margin-left: -0.019ex; width:8.736ex; height:5.843ex;" alt="{\displaystyle \sum _{j\in i+N}v_{w_{j}}}" loading="lazy"></span> is the sum of a word's neighbors' vectors, etc.

</p><p>Once such a system is trained, we have two trained matrices <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V,V'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V,V'}</annotation>

</semantics>

</math></span><img src="./f7beab57ca0526a8b51dc1f1df846019d134452d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.423ex; height:2.843ex;" alt="{\displaystyle V,V'}" loading="lazy"></span>. Either the column vectors of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span> or the row vectors of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V'}</annotation>

</semantics>

</math></span><img src="./ff31fe992a31b4c954a933f1be91e2739a1c0ae7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.602ex; height:2.509ex;" alt="{\displaystyle V'}" loading="lazy"></span> can serve as the dictionary. For example, the word "sat" can be represented as either the "sat"-th column of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span> or the "sat"-th row of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V'}</annotation>

</semantics>

</math></span><img src="./ff31fe992a31b4c954a933f1be91e2739a1c0ae7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.602ex; height:2.509ex;" alt="{\displaystyle V'}" loading="lazy"></span>. It is also possible to simply define <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V'=V^{\top }}">

<msup>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V'=V^{\top }}</annotation>

</semantics>

</math></span><img src="./f0ae890beef3ec8927faeac2d34a2baea5a18dea.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:9.128ex; height:2.676ex;" alt="{\displaystyle V'=V^{\top }}" loading="lazy"></span>, in which case there would no longer be a choice.

</p>

<p>The idea of skip-gram is to represent each word with a vector, such that it is possible to predict the vectors of its neighbors using the vector of a word.

</p><p>The architecture is still linear-linear-softmax, the same as CBOW, but the input and the output are switched. Specifically, for each word <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation>

</semantics>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation>

</semantics>

</math></span><img src="./fe22f0329d3ecb2e1880d44d191aba0e5475db68.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" loading="lazy"></span>. The objective of training is to maximize <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sum _{i}\sum _{j\in i+N}\ln \Pr(w_{j}|w_{i})}">

</mrow>

</munder>

</mrow>

</munder>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sum _{i}\sum _{j\in i+N}\ln \Pr(w_{j}|w_{i})}</annotation>

</semantics>

</math></span><img src="./92ff8c39af1c5ec7c60bbe29b71f6ea6e83745a8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:21.523ex; height:5.843ex;" alt="{\displaystyle \sum _{i}\sum _{j\in i+N}\ln \Pr(w_{j}|w_{i})}" loading="lazy"></span>.

</p><p>In full formula, the loss function is<span class="mwe-math-element mwe-math-element-block"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle -\sum _{i}\sum _{j\in i+N}\ln {\frac {e^{v_{w_{j}}'\cdot v_{w_{i}}}}{\sum _{w'}e^{v_{w'}'\cdot v_{w_{i}}}}}}">

</mrow>

</munder>

</mrow>

</munder>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msubsup>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

<msup>

</msup>

</mrow>

</munder>

<msup>

<msup>

</msup>

</mrow>

</msubsup>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle -\sum _{i}\sum _{j\in i+N}\ln {\frac {e^{v_{w_{j}}'\cdot v_{w_{i}}}}{\sum _{w'}e^{v_{w'}'\cdot v_{w_{i}}}}}}</annotation>

</semantics>

</math></span></span>Same as CBOW, once such a system is trained, we have two trained matrices <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V,V'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V,V'}</annotation>

</semantics>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V}</annotation>

</semantics>

</math></span><img src="./af0f6064540e84211d0ffe4dac72098adfa52845.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.787ex; height:2.176ex;" alt="{\displaystyle V}" loading="lazy"></span> or the row vectors of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V'}</annotation>

</semantics>

</math></span><img src="./ff31fe992a31b4c954a933f1be91e2739a1c0ae7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.602ex; height:2.509ex;" alt="{\displaystyle V'}" loading="lazy"></span> can serve as the dictionary. It is also possible to simply define <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V'=V^{\top }}">

<msup>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V'=V^{\top }}</annotation>

</semantics>

</p><p>Essentially, skip-gram and CBOW are exactly the same in architecture. They only differ in the objective function during training.

</p>

<div class="mw-heading mw-heading2"><h2 id="History">History</h2></div>

<p>During the 1980s, there were some early attempts at using neural networks to represent words and concepts as vectors.<sup id="cite_ref-6" class="reference"><a href="#cite_note-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup>

</p><p>In 2010, <a href="Tom%C3%A1%C5%A1_Mikolov" title="Tomáš Mikolov">Tomáš Mikolov</a> (then at <a href="Brno_University_of_Technology" title="Brno University of Technology">Brno University of Technology</a>) with co-authors applied a simple <a href="Recurrent_neural_network" title="Recurrent neural network">recurrent neural network</a> with a single hidden layer to language modelling.<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup>

</p><p>Word2vec was created, patented,<sup id="cite_ref-pat_10-0" class="reference"><a href="#cite_note-pat-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> and published in 2013 by a team of researchers led by Mikolov at <a href="Google" title="Google">Google</a> over two papers.<sup id="cite_ref-mikolov_1-4" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-mikolov-nips_2-2" class="reference"><a href="#cite_note-mikolov-nips-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> The original paper was rejected by reviewers for <a href="International_Conference_on_Learning_Representations" title="International Conference on Learning Representations">ICLR conference</a> 2013. It also took months for the code to be approved for open-sourcing.<sup id="cite_ref-:4_11-0" class="reference"><a href="#cite_note-:4-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> Other researchers helped analyse and explain the algorithm.<sup id="cite_ref-explain_4-1" class="reference"><a href="#cite_note-explain-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup>

</p><p>Embedding vectors created using the Word2vec algorithm have some advantages compared to earlier algorithms<sup id="cite_ref-mikolov_1-5" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> such as those using n-grams and <a href="Latent_semantic_analysis" title="Latent semantic analysis">latent semantic analysis</a>. <a href="GloVe" title="GloVe">GloVe</a> was developed by a team at Stanford specifically as a competitor, and the original paper noted multiple improvements of GloVe over word2vec.<sup id="cite_ref-12" class="reference"><a href="#cite_note-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> Mikolov argued that the comparison was unfair as GloVe was trained on more data, and that the <a href="FastText" title="FastText">fastText</a> project showed that word2vec is superior when trained on the same data.<sup id="cite_ref-13" class="reference"><a href="#cite_note-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:4_11-1" class="reference"><a href="#cite_note-:4-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup>

</p><p>As of 2022, the straight Word2vec approach was described as "dated". <a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>-based models, such as <a href="ELMo" title="ELMo">ELMo</a> and <a href="BERT_(language_model)" title="BERT (language model)">BERT</a>, which add multiple neural-network attention layers on top of a word embedding model similar to Word2vec, have come to be regarded as the state of the art in NLP.<sup id="cite_ref-14" class="reference"><a href="#cite_note-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Parameterization">Parameterization</h2></div>

<p>Results of word2vec training can be sensitive to <a href="Parameter" title="Parameter">parametrization</a>. The following are some important parameters in word2vec training.

</p>

<div class="mw-heading mw-heading3"><h3 id="Training_algorithm">Training algorithm</h3></div>

<p>A Word2vec model can be trained with hierarchical <a href="Softmax_function" title="Softmax function">softmax</a> and/or negative sampling. To approximate the conditional log-likelihood a model seeks to maximize, the hierarchical softmax method uses a <a href="Huffman_coding" title="Huffman coding">Huffman tree</a> to reduce calculation. The negative sampling method, on the other hand, approaches the maximization problem by minimizing the <a href="Log-likelihood" class="mw-redirect" title="Log-likelihood">log-likelihood</a> of sampled negative instances. According to the authors, hierarchical softmax works better for infrequent words while negative sampling works better for frequent words and better with low dimensional vectors.<sup id="cite_ref-:1_3-1" class="reference"><a href="#cite_note-:1-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> As training epochs increase, hierarchical softmax stops being useful.<sup id="cite_ref-15" class="reference"><a href="#cite_note-15"><span class="cite-bracket">[</span>15<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading3"><h3 id="Sub-sampling">Sub-sampling</h3></div>

<p>High-frequency and low-frequency words often provide little information. Words with a frequency above a certain threshold, or below a certain threshold, may be subsampled or removed to speed up training.<sup id="cite_ref-16" class="reference"><a href="#cite_note-16"><span class="cite-bracket">[</span>16<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading3"><h3 id="Dimensionality">Dimensionality</h3></div>

<p>Quality of word embedding increases with higher dimensionality. But after reaching some point, marginal gain diminishes.<sup id="cite_ref-mikolov_1-6" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> Typically, the dimensionality of the vectors is set to be between 100 and 1,000.

</p>

<div class="mw-heading mw-heading3"><h3 id="Context_window">Context window</h3></div>

<p>The size of the context window determines how many words before and after a given word are included as context words of the given word. According to the authors' note, the recommended value is 10 for skip-gram and 5 for CBOW.<sup id="cite_ref-:1_3-2" class="reference"><a href="#cite_note-:1-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Extensions">Extensions</h2></div>

<p>There are a variety of extensions to word2vec.

</p>

<p>doc2vec, generates distributed representations of <i>variable-length</i> pieces of texts, such as sentences, paragraphs, or entire documents.<sup id="cite_ref-:2_17-0" class="reference"><a href="#cite_note-:2-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-18" class="reference"><a href="#cite_note-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup> doc2vec has been implemented in the <a href="C_(programming_language)" title="C (programming language)">C</a>, <a href="Python_(programming_language)" title="Python (programming language)">Python</a> and <a href="Java_(programming_language)" title="Java (programming language)">Java</a>/<a href="Scala_(programming_language)" title="Scala (programming language)">Scala</a> tools (see below), with the Java and Python versions also supporting inference of document embeddings on new, unseen documents.

</p><p>doc2vec estimates the distributed representations of documents much like how word2vec estimates representations of words: doc2vec utilizes either of two model architectures, both of which are allegories to the architectures used in word2vec. The first, Distributed Memory Model of Paragraph Vectors (PV-DM), is identical to CBOW other than it also provides a unique document identifier as a piece of additional context. The second architecture, Distributed Bag of Words version of Paragraph Vector (PV-DBOW), is identical to the skip-gram model except that it attempts to predict the window of surrounding context words from the paragraph identifier instead of the current word.<sup id="cite_ref-:2_17-1" class="reference"><a href="#cite_note-:2-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup>

</p><p>doc2vec also has the ability to capture the semantic ‘meanings’ for additional pieces of  ‘context’ around words; doc2vec can estimate the semantic embeddings for speakers or speaker attributes, groups, and periods of time. For example, doc2vec has been used to estimate the political positions of political parties in various Congresses and Parliaments in the U.S. and U.K.,<sup id="cite_ref-19" class="reference"><a href="#cite_note-19"><span class="cite-bracket">[</span>19<span class="cite-bracket">]</span></a></sup> respectively, and various governmental institutions.<sup id="cite_ref-20" class="reference"><a href="#cite_note-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup>

</p>

<p>Another extension of word2vec is top2vec, which leverages both document and word embeddings to estimate distributed representations of topics.<sup id="cite_ref-:3_21-0" class="reference"><a href="#cite_note-:3-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-22" class="reference"><a href="#cite_note-22"><span class="cite-bracket">[</span>22<span class="cite-bracket">]</span></a></sup> top2vec takes document embeddings learned from a doc2vec model and <a href="Dimensionality_reduction" title="Dimensionality reduction">reduces</a> them into a lower dimension (typically using <a href="Uniform_Manifold_Approximation_and_Projection" class="mw-redirect" title="Uniform Manifold Approximation and Projection">UMAP</a>). The space of documents is then scanned using <a href="HDBSCAN" class="mw-redirect" title="HDBSCAN">HDBSCAN</a>,<sup id="cite_ref-23" class="reference"><a href="#cite_note-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup> and clusters of similar documents are found. Next, the centroid of documents identified in a cluster is considered to be that cluster's topic vector. Finally, top2vec searches the semantic space for word embeddings located near to the topic vector to ascertain the 'meaning' of the topic.<sup id="cite_ref-:3_21-1" class="reference"><a href="#cite_note-:3-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup> The word with embeddings most similar to the topic vector might be assigned as the topic's title, whereas far away word embeddings may be considered unrelated.

</p><p>As opposed to other topic models such as <a href="Latent_Dirichlet_allocation" title="Latent Dirichlet allocation">LDA</a>, top2vec provides canonical ‘distance’ metrics between two topics, or between a topic and another embeddings (word, document, or otherwise). Together with results from HDBSCAN, users can generate topic hierarchies, or groups of related topics and subtopics.

</p><p>Furthermore, a user can use the results of top2vec to infer the topics of out-of-sample documents. After inferring the embedding for a new document, must only search the space of topics for the closest topic vector.

</p>

<div class="mw-heading mw-heading3"><h3 id="BioVectors">BioVectors</h3></div>

<p>An extension of word vectors for n-grams in <a href="Biological" class="mw-redirect" title="Biological">biological</a> sequences (e.g. <a href="DNA" title="DNA">DNA</a>, <a href="RNA" title="RNA">RNA</a>, and <a href="Protein" title="Protein">proteins</a>) for <a href="Bioinformatic" class="mw-redirect" title="Bioinformatic">bioinformatics</a> applications has been proposed by Asgari and Mofrad.<sup id="cite_ref-:0_24-0" class="reference"><a href="#cite_note-:0-24"><span class="cite-bracket">[</span>24<span class="cite-bracket">]</span></a></sup> Named bio-vectors (BioVec) to refer to biological sequences in general with protein-vectors (ProtVec) for proteins (amino-acid sequences) and gene-vectors (GeneVec) for gene sequences, this representation can be widely used in applications of machine learning in proteomics and genomics. The results suggest that BioVectors can characterize biological sequences in terms of biochemical and biophysical interpretations of the underlying patterns.<sup id="cite_ref-:0_24-1" class="reference"><a href="#cite_note-:0-24"><span class="cite-bracket">[</span>24<span class="cite-bracket">]</span></a></sup> A similar variant, dna2vec, has shown that there is correlation between <a href="Needleman%E2%80%93Wunsch_algorithm" title="Needleman–Wunsch algorithm">Needleman–Wunsch</a> similarity score and <a href="Cosine_similarity" title="Cosine similarity">cosine similarity</a> of dna2vec word vectors.<sup id="cite_ref-25" class="reference"><a href="#cite_note-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading3"><h3 id="Radiology_and_intelligent_word_embeddings_(IWE)">Radiology and intelligent word embeddings (IWE)</h3></div>

<p>An extension of word vectors for creating a dense vector representation of unstructured radiology reports has been proposed by Banerjee et al.<sup id="cite_ref-26" class="reference"><a href="#cite_note-26"><span class="cite-bracket">[</span>26<span class="cite-bracket">]</span></a></sup> One of the biggest challenges with Word2vec is how to handle unknown or out-of-vocabulary (OOV) words and morphologically similar words. If the Word2vec model has not encountered a particular word before, it will be forced to use a random vector, which is generally far from its ideal representation. This can particularly be an issue in domains like medicine where synonyms and related words can be used depending on the preferred style of radiologist, and words may have been used infrequently in a large corpus.

</p><p>IWE combines Word2vec with a semantic dictionary mapping technique to tackle the major challenges of <a href="Information_extraction" title="Information extraction">information extraction</a> from clinical texts, which include ambiguity of free text narrative style, lexical variations, use of ungrammatical and telegraphic phases, arbitrary ordering of words, and frequent appearance of abbreviations and acronyms. Of particular interest, the IWE model (trained on the one institutional dataset) successfully translated to a different institutional dataset which demonstrates good generalizability of the approach across institutions.

</p>

<div class="mw-heading mw-heading2"><h2 id="Analysis">Analysis</h2></div>

<p>The reasons for successful <a href="Word_embedding" title="Word embedding">word embedding</a> learning in the word2vec framework are poorly understood. Goldberg and Levy point out that the word2vec objective function causes words that occur in similar contexts to have similar embeddings (as measured by <a href="Cosine_similarity" title="Cosine similarity">cosine similarity</a>) and note that this is in line with J. R. Firth's <a href="Distributional_semantics" title="Distributional semantics">distributional hypothesis</a>. However, they note that this explanation is "very hand-wavy" and argue that a more formal explanation would be preferable.<sup id="cite_ref-explain_4-2" class="reference"><a href="#cite_note-explain-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup>

</p><p>Levy et al. (2015)<sup id="cite_ref-27" class="reference"><a href="#cite_note-27"><span class="cite-bracket">[</span>27<span class="cite-bracket">]</span></a></sup> show that much of the superior performance of word2vec or similar embeddings in downstream tasks is not a result of the models per se, but of the choice of specific hyperparameters. Transferring these hyperparameters to more 'traditional' approaches yields similar performances in downstream tasks. Arora et al. (2016)<sup id="cite_ref-28" class="reference"><a href="#cite_note-28"><span class="cite-bracket">[</span>28<span class="cite-bracket">]</span></a></sup> explain word2vec and related algorithms as performing inference for a simple <a href="Generative_model" title="Generative model">generative model</a> for text, which involves a random walk generation process based upon loglinear topic model. They use this to explain some properties of word embeddings, including their use to solve analogies.

</p>

<div class="mw-heading mw-heading2"><h2 id="Preservation_of_semantic_and_syntactic_relationships">Preservation of semantic and syntactic relationships</h2></div>

<p>The word embedding approach is able to capture multiple different degrees of similarity between words. Mikolov et al. (2013)<sup id="cite_ref-29" class="reference"><a href="#cite_note-29"><span class="cite-bracket">[</span>29<span class="cite-bracket">]</span></a></sup> found that semantic and syntactic patterns can be reproduced using vector arithmetic. Patterns such as "Man is to Woman as Brother is to Sister" can be generated through algebraic operations on the vector representations of these words such that the vector representation of "Brother" - "Man" + "Woman" produces a result which is closest to the vector representation of "Sister" in the model. Such relationships can be generated for a range of semantic relations (such as Country–Capital) as well as syntactic relations (e.g. present tense–past tense).

</p><p>This facet of word2vec has been exploited in a variety of other contexts. For example, word2vec has been used to map a vector space of words in one language to a vector space constructed from another language. Relationships between translated words in both spaces can be used to assist with <a href="Machine_translation" title="Machine translation">machine translation</a> of new words.<sup id="cite_ref-30" class="reference"><a href="#cite_note-30"><span class="cite-bracket">[</span>30<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Assessing_the_quality_of_a_model">Assessing the quality of a model</h2></div>

<p>Mikolov et al. (2013)<sup id="cite_ref-mikolov_1-7" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> developed an approach to assessing the quality of a word2vec model which draws on the semantic and syntactic patterns discussed above. They developed a set of 8,869 semantic relations and 10,675 syntactic relations which they use as a benchmark to test the accuracy of a model. When assessing the quality of a vector model, a user may draw on this accuracy test which is implemented in word2vec,<sup id="cite_ref-31" class="reference"><a href="#cite_note-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup> or develop their own test set which is meaningful to the corpora which make up the model. This approach offers a more challenging test than simply arguing that the words most similar to a given test word are intuitively plausible.<sup id="cite_ref-mikolov_1-8" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading3"><h3 id="Parameters_and_model_quality">Parameters and model quality</h3></div>

<p>The use of different model parameters and different corpus sizes can greatly affect the quality of a word2vec model. Accuracy can be improved in a number of ways, including the choice of model architecture (CBOW or Skip-Gram), increasing the training data set, increasing the number of vector dimensions, and increasing the window size of words considered by the algorithm. Each of these improvements comes with the cost of increased computational complexity and therefore increased model generation time.<sup id="cite_ref-mikolov_1-9" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup>

</p><p>In models using large corpora and a high number of dimensions, the skip-gram model yields the highest overall accuracy, and consistently produces the highest accuracy on semantic relationships, as well as yielding the highest syntactic accuracy in most cases. However, the CBOW is less computationally expensive and yields similar accuracy results.<sup id="cite_ref-mikolov_1-10" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup>

</p><p>Overall, accuracy increases with the number of words used and the number of dimensions. Mikolov et al.<sup id="cite_ref-mikolov_1-11" class="reference"><a href="#cite_note-mikolov-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> report that doubling the amount of training data results in an increase in computational complexity equivalent to doubling the number of vector dimensions.

</p><p>Altszyler and coauthors (2017) studied Word2vec performance in two semantic tests for different corpus size.<sup id="cite_ref-Altszyler_32-0" class="reference"><a href="#cite_note-Altszyler-32"><span class="cite-bracket">[</span>32<span class="cite-bracket">]</span></a></sup> They found that Word2vec has a steep <a href="Learning_curve_(machine_learning)" title="Learning curve (machine learning)">learning curve</a>, outperforming another word-embedding technique, <a href="Latent_semantic_analysis" title="Latent semantic analysis">latent semantic analysis</a> (LSA), when it is trained with medium to large corpus size (more than 10 million words). However, with a small training corpus, LSA showed better performance. Additionally they show that the best parameter setting depends on the task and the training corpus. Nevertheless, for skip-gram models trained in medium size corpora, with 50 dimensions, a window size of 15 and 10 negative samples seems to be a good parameter setting.

</p>

/* start https://en.wikipedia.org/ */

.mw-parser-output .div-col{margin-top:0.3em;column-width:30em}.mw-parser-output .div-col-small{font-size:90%}.mw-parser-output .div-col-rules{column-rule:1px solid #aaa}.mw-parser-output .div-col dl,.mw-parser-output .div-col ol,.mw-parser-output .div-col ul{margin-top:0}.mw-parser-output .div-col li,.mw-parser-output .div-col dd{page-break-inside:avoid;break-inside:avoid-column}

/* end https://en.wikipedia.org/ */

</style><div class="div-col" style="column-width: 20em;">

<ul><li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Document-term_matrix" title="Document-term matrix">Document-term matrix</a></li>

<li><a href="Feature_extraction" class="mw-redirect" title="Feature extraction">Feature extraction</a></li>

<li><a href="Feature_learning" title="Feature learning">Feature learning</a></li>

<li><a href="Language_model#Neural_models" title="Language model">Language model § Neural models</a></li>

<li><a href="Vector_space_model" title="Vector space model">Vector space model</a></li>

<li><a href="Thought_vector" title="Thought vector">Thought vector</a></li>

<li><a href="FastText" title="FastText">fastText</a></li>

<li><a href="GloVe" title="GloVe">GloVe</a></li>

<li><a href="BERT_(language_model)" title="BERT (language model)">BERT (language model)</a></li>

<li><a href="Normalized_compression_distance" title="Normalized compression distance">Normalized compression distance</a></li></ul>

</div>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist">

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFMikolovChenCorradoDean2013" class="citation arxiv cs1">Mikolov, Tomas; Chen, Kai; Corrado, Greg; Dean, Jeffrey (16 January 2013). "Efficient Estimation of Word Representations in Vector Space". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1301.3781">1301.3781</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite></span>

</li>

<li id="cite_note-mikolov-nips-2"><span class="mw-cite-backlink">^ <a href="#cite_ref-mikolov-nips_2-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-mikolov-nips_2-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-mikolov-nips_2-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFMikolovSutskeverChenCorrado2013" class="citation conference cs1">Mikolov, Tomas; Sutskever, Ilya; Chen, Kai; Corrado, Greg S.; Dean, Jeff (2013). <i>Distributed representations of words and phrases and their compositionality</i>. <a href="Advances_in_Neural_Information_Processing_Systems" class="mw-redirect" title="Advances in Neural Information Processing Systems">Advances in Neural Information Processing Systems</a>. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1310.4546">1310.4546</a></span>. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2013arXiv1310.4546M">2013arXiv1310.4546M</a>.</cite></span>

</li>

<li id="cite_note-:1-3"><span class="mw-cite-backlink">^ <a href="#cite_ref-:1_3-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:1_3-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:1_3-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://code.google.com/archive/p/word2vec/">"Google Code Archive - Long-term storage for Google Code Project Hosting"</a>. <i>code.google.com</i><span class="reference-accessdate">. Retrieved <span class="nowrap">13 June</span> 2016</span>.</cite></span>

</li>

<li id="cite_note-explain-4"><span class="mw-cite-backlink">^ <a href="#cite_ref-explain_4-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-explain_4-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-explain_4-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFGoldbergLevy2014" class="citation arxiv cs1">Goldberg, Yoav; Levy, Omer (2014). "word2vec Explained: Deriving Mikolov et al.'s Negative-Sampling Word-Embedding Method". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1402.3722">1402.3722</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite></span>

</li>

<li id="cite_note-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-5">^</a></b></span> <span class="reference-text"><cite id="CITEREFRong2016" class="citation cs2">Rong, Xin (5 June 2016), <i>word2vec Parameter Learning Explained</i>, <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1411.2738">1411.2738</a></span></cite></span>

</li>

<li id="cite_note-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-6">^</a></b></span> <span class="reference-text">Hinton, Geoffrey E. "Learning distributed representations of concepts." <i>Proceedings of the Annual Meeting of the Cognitive Science Society</i>. Vol. 8. 1986.</span>

</li>

<li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><cite id="CITEREFRumelhartMcClelland1985" class="citation report cs1">Rumelhart, David E.; McClelland, James L. (October 1985). <a rel="nofollow" class="external text" href="https://eric.ed.gov/?id=ED267419">On Learning the Past Tenses of English Verbs</a> (Report).</cite></span>

</li>

<li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><cite id="CITEREFElman1990" class="citation journal cs1">Elman, Jeffrey L. (1 April 1990). <a rel="nofollow" class="external text" href="https://www.sciencedirect.com/science/article/abs/pii/036402139090002E">"Finding structure in time"</a>. <i>Cognitive Science</i>. <b>14</b> (2): <span class="nowrap">179–</span>211. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0364-0213%2890%2990002-E">10.1016/0364-0213(90)90002-E</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0364-0213">0364-0213</a>.</cite></span>

</li>

<li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><cite id="CITEREFMikolovKarafiátBurgetČernocký2010" class="citation conference cs1">Mikolov, Tomáš; Karafiát, Martin; Burget, Lukáš; Černocký, Jan; Khudanpur, Sanjeev (26 September 2010). "Recurrent neural network based language model". <i>Interspeech 2010</i>. ISCA: ISCA. pp. <span class="nowrap">1045–</span>1048. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.21437%2Finterspeech.2010-343">10.21437/interspeech.2010-343</a>.</cite></span>

</li>

/* start https://en.wikipedia.org/ */

.mw-parser-output .citation{word-wrap:break-word}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}

/* end https://en.wikipedia.org/ */

</style><span class="citation patent" id="CITEREFMikolovChenCorradoDean2015"><a rel="nofollow" class="external text" href="https://worldwide.espacenet.com/textdoc?DB=EPODOC&IDX=US9037464">US 9037464</a>, Mikolov, Tomas; Chen, Kai & Corrado, Gregory S. et al., "Computing numeric representations of words in a high-dimensional space", published 19 May 2015, assigned to <a href="Google_Inc." class="mw-redirect" title="Google Inc.">Google Inc.</a></span><span class="Z3988" title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Apatent&rft.number=9037464&rft.cc=US&rft.title=Computing+numeric+representations+of+words+in+a+high-dimensional+space&rft.inventor=Mikolov&rft.assignee=%5B%5BGoogle+Inc.%5D%5D&rft.pubdate=2015-05-19"><span style="display: none;"> </span></span></span>

</li>

<li id="cite_note-:4-11"><span class="mw-cite-backlink">^ <a href="#cite_ref-:4_11-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:4_11-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFMikolov2023" class="citation web cs1">Mikolov, Tomáš (13 December 2023). <a rel="nofollow" class="external text" href="https://www.facebook.com/tomas.mikolov/posts/yesterday-we-received-a-test-of-time-award-at-neurips-for-the-word2vec-paper-fro/10231641326366223/">"Yesterday we received a Test of Time Award at NeurIPS for the word2vec paper from ten years ago"</a>. <i><a href="Facebook" title="Facebook">Facebook</a></i>. <a rel="nofollow" class="external text" href="https://archive.today/20231224074628/https://news.ycombinator.com/item?id=38654038">Archived</a> from the original on 24 December 2023.</cite></span>

</li>

<li id="cite_note-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-12">^</a></b></span> <span class="reference-text"><a rel="nofollow" class="external text" href="https://www.aclweb.org/anthology/D14-1162"><i>GloVe: Global Vectors for Word Representation</i> (pdf)</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20200903022100/https://www.aclweb.org/anthology/D14-1162/">Archived</a> 2020-09-03 at the <a href="Wayback_Machine" title="Wayback Machine">Wayback Machine</a> "We use our insights to construct a new model for word representation which we call GloVe, for Global Vectors, because the global corpus statistics are captured directly by the model."</span>

</li>

<li id="cite_note-13"><span class="mw-cite-backlink"><b><a href="#cite_ref-13">^</a></b></span> <span class="reference-text"><cite id="CITEREFJoulinGraveBojanowskiMikolov2016" class="citation arxiv cs1">Joulin, Armand; Grave, Edouard; Bojanowski, Piotr; Mikolov, Tomas (9 August 2016). "Bag of Tricks for Efficient Text Classification". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1607.01759">1607.01759</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite></span>

</li>

<li id="cite_note-14"><span class="mw-cite-backlink"><b><a href="#cite_ref-14">^</a></b></span> <span class="reference-text"><cite id="CITEREFVon_der_MoselTrautschHerbold2022" class="citation journal cs1">Von der Mosel, Julian; Trautsch, Alexander; Herbold, Steffen (2022). <a rel="nofollow" class="external text" href="https://ieeexplore.ieee.org/document/9785808">"On the validity of pre-trained transformers for natural language processing in the software engineering domain"</a>. <i>IEEE Transactions on Software Engineering</i>. <b>49</b> (4): <span class="nowrap">1487–</span>1507. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2109.04738">2109.04738</a></span>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTSE.2022.3178469">10.1109/TSE.2022.3178469</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1939-3520">1939-3520</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:237485425">237485425</a>.</cite></span>

</li>

<li id="cite_note-15"><span class="mw-cite-backlink"><b><a href="#cite_ref-15">^</a></b></span> <span class="reference-text"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://groups.google.com/forum/#!msg/word2vec-toolkit/WUWad9fL0jU/LdbWy1jQjUIJ">"Parameter (hs & negative)"</a>. <i>Google Groups</i><span class="reference-accessdate">. Retrieved <span class="nowrap">13 June</span> 2016</span>.</cite></span>

</li>

<li id="cite_note-16"><span class="mw-cite-backlink"><b><a href="#cite_ref-16">^</a></b></span> <span class="reference-text"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="http://jmlr.csail.mit.edu/papers/volume9/vandermaaten08a/vandermaaten08a.pdf">"Visualizing Data using t-SNE"</a> <span class="cs1-format">(PDF)</span>. <i>Journal of Machine Learning Research, 2008. Vol. 9, pg. 2595</i><span class="reference-accessdate">. Retrieved <span class="nowrap">18 March</span> 2017</span>.</cite></span>

</li>

<li id="cite_note-:2-17"><span class="mw-cite-backlink">^ <a href="#cite_ref-:2_17-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:2_17-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFLeMikolov2014" class="citation journal cs1">Le, Quoc; Mikolov, Tomas (May 2014). "Distributed Representations of Sentences and Documents". <i>Proceedings of the 31st International Conference on Machine Learning</i>. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1405.4053">1405.4053</a></span>.</cite></span>

</li>

<li id="cite_note-18"><span class="mw-cite-backlink"><b><a href="#cite_ref-18">^</a></b></span> <span class="reference-text"><cite id="CITEREFRehurek" class="citation web cs1">Rehurek, Radim. <a rel="nofollow" class="external text" href="https://radimrehurek.com/gensim/models/doc2vec.html">"Gensim"</a>.</cite></span>

</li>

<li id="cite_note-19"><span class="mw-cite-backlink"><b><a href="#cite_ref-19">^</a></b></span> <span class="reference-text"><cite id="CITEREFRheaultCochrane2019" class="citation journal cs1">Rheault, Ludovic; Cochrane, Christopher (3 July 2019). <a rel="nofollow" class="external text" href="https://www.cambridge.org/core/journals/political-analysis/article/word-embeddings-for-the-analysis-of-ideological-placement-in-parliamentary-corpora/017F0CEA9B3DB6E1B94AC36A509A8A7B">"Word Embeddings for the Analysis of Ideological Placement in Parliamentary Corpora"</a>. <i>Political Analysis</i>. <b>28</b> (1).</cite></span>

</li>

<li id="cite_note-20"><span class="mw-cite-backlink"><b><a href="#cite_ref-20">^</a></b></span> <span class="reference-text"><cite id="CITEREFNay2017" class="citation journal cs1">Nay, John (21 December 2017). <a rel="nofollow" class="external text" href="https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3087278">"Gov2Vec: Learning Distributed Representations of Institutions and Their Legal Text"</a>. <i>SSRN</i>. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1609.06616">1609.06616</a></span>. <a href="SSRN_(identifier)" class="mw-redirect" title="SSRN (identifier)">SSRN</a> <a rel="nofollow" class="external text" href="https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3087278">3087278</a>.</cite></span>

</li>

<li id="cite_note-:3-21"><span class="mw-cite-backlink">^ <a href="#cite_ref-:3_21-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:3_21-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFAngelov2020" class="citation arxiv cs1">Angelov, Dimo (August 2020). "Top2Vec: Distributed Representations of Topics". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2008.09470">2008.09470</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite></span>

</li>

<li id="cite_note-22"><span class="mw-cite-backlink"><b><a href="#cite_ref-22">^</a></b></span> <span class="reference-text"><cite id="CITEREFAngelov2022" class="citation web cs1">Angelov, Dimo (11 November 2022). <a rel="nofollow" class="external text" href="https://github.com/ddangelov/Top2Vec">"Top2Vec"</a>. <i><a href="GitHub" title="GitHub">GitHub</a></i>.</cite></span>

</li>

<li id="cite_note-23"><span class="mw-cite-backlink"><b><a href="#cite_ref-23">^</a></b></span> <span class="reference-text"><cite id="CITEREFCampelloMoulaviSander2013" class="citation book cs1">Campello, Ricardo; Moulavi, Davoud; Sander, Joerg (2013). <a rel="nofollow" class="external text" href="https://link.springer.com/chapter/10.1007/978-3-642-37456-2_14">"Density-Based Clustering Based on Hierarchical Density Estimates"</a>. <i>Advances in Knowledge Discovery and Data Mining</i>. Lecture Notes in Computer Science. Vol. 7819. pp. <span class="nowrap">160–</span>172. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-642-37456-2_14">10.1007/978-3-642-37456-2_14</a>. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-3-642-37455-5</bdi>.</cite></span>

</li>

<li id="cite_note-:0-24"><span class="mw-cite-backlink">^ <a href="#cite_ref-:0_24-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:0_24-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFAsgariMofrad2015" class="citation journal cs1">Asgari, Ehsaneddin; Mofrad, Mohammad R.K. (2015). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4640716">"Continuous Distributed Representation of Biological Sequences for Deep Proteomics and Genomics"</a>. <i>PLOS ONE</i>. <b>10</b> (11): e0141287. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1503.05140">1503.05140</a></span>. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2015PLoSO..1041287A">2015PLoSO..1041287A</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1371%2Fjournal.pone.0141287">10.1371/journal.pone.0141287</a></span>. <a href="PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4640716">4640716</a></span>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/26555596">26555596</a>.</cite></span>

</li>

<li id="cite_note-25"><span class="mw-cite-backlink"><b><a href="#cite_ref-25">^</a></b></span> <span class="reference-text"><cite id="CITEREFNg2017" class="citation arxiv cs1">Ng, Patrick (2017). "dna2vec: Consistent vector representations of variable-length k-mers". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1701.06279">1701.06279</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/q-bio.QM">q-bio.QM</a>].</cite></span>

</li>

<li id="cite_note-26"><span class="mw-cite-backlink"><b><a href="#cite_ref-26">^</a></b></span> <span class="reference-text"><cite id="CITEREFBanerjeeChenLungrenRubin2018" class="citation journal cs1">Banerjee, Imon; Chen, Matthew C.; Lungren, Matthew P.; Rubin, Daniel L. (2018). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5771955">"Radiology report annotation using intelligent word embeddings: Applied to multi-institutional chest CT cohort"</a>. <i>Journal of Biomedical Informatics</i>. <b>77</b>: <span class="nowrap">11–</span>20. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.jbi.2017.11.012">10.1016/j.jbi.2017.11.012</a>. <a href="PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5771955">5771955</a></span>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/29175548">29175548</a>.</cite></span>

</li>

<li id="cite_note-27"><span class="mw-cite-backlink"><b><a href="#cite_ref-27">^</a></b></span> <span class="reference-text"><cite id="CITEREFLevyGoldbergDagan2015" class="citation journal cs1">Levy, Omer; Goldberg, Yoav; Dagan, Ido (2015). <a rel="nofollow" class="external text" href="http://www.aclweb.org/anthology/Q15-1016">"Improving Distributional Similarity with Lessons Learned from Word Embeddings"</a>. <i>Transactions of the Association for Computational Linguistics</i>. <b>3</b>. Transactions of the Association for Computational Linguistics: <span class="nowrap">211–</span>225. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Ftacl_a_00134">10.1162/tacl_a_00134</a></span>.</cite></span>

</li>

<li id="cite_note-28"><span class="mw-cite-backlink"><b><a href="#cite_ref-28">^</a></b></span> <span class="reference-text"><cite id="CITEREFArora2016" class="citation journal cs1">Arora, S; et al. (Summer 2016). <a rel="nofollow" class="external text" href="http://aclweb.org/anthology/Q16-1028">"A Latent Variable Model Approach to PMI-based Word Embeddings"</a>. <i>Transactions of the Association for Computational Linguistics</i>. <b>4</b>: <span class="nowrap">385–</span>399. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1502.03520">1502.03520</a></span>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Ftacl_a_00106">10.1162/tacl_a_00106</a></span> – via ACLWEB.</cite></span>

</li>

<li id="cite_note-29"><span class="mw-cite-backlink"><b><a href="#cite_ref-29">^</a></b></span> <span class="reference-text"><cite id="CITEREFMikolovYihZweig2013" class="citation journal cs1">Mikolov, Tomas; Yih, Wen-tau; Zweig, Geoffrey (2013). "Linguistic Regularities in Continuous Space Word Representations". <i>HLT-Naacl</i>: <span class="nowrap">746–</span>751.</cite></span>

</li>

<li id="cite_note-30"><span class="mw-cite-backlink"><b><a href="#cite_ref-30">^</a></b></span> <span class="reference-text"><cite id="CITEREFJansen2017" class="citation web cs1">Jansen, Stefan (9 May 2017). "Word and Phrase Translation with word2vec". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1705.03127">1705.03127</a></span>.</cite> <span class="cs1-visible-error citation-comment"><code class="cs1-code">{{cite web}}</code>: </span><span class="cs1-visible-error citation-comment">Missing or empty <code class="cs1-code">|url=</code> (help)</span></span>

</li>

<li id="cite_note-31"><span class="mw-cite-backlink"><b><a href="#cite_ref-31">^</a></b></span> <span class="reference-text"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://radimrehurek.com/gensim/models/word2vec.html">"Gensim - Deep learning with word2vec"</a><span class="reference-accessdate">. Retrieved <span class="nowrap">10 June</span> 2016</span>.</cite></span>

</li>

<li id="cite_note-Altszyler-32"><span class="mw-cite-backlink"><b><a href="#cite_ref-Altszyler_32-0">^</a></b></span> <span class="reference-text"><cite id="CITEREFAltszyler,_E.Ribeiro,_S.Sigman,_M.Fernández_Slezak,_D.2017" class="citation journal cs1">Altszyler, E.; Ribeiro, S.; Sigman, M.; Fernández Slezak, D. (2017). "The interpretation of dream meaning: Resolving ambiguity using Latent Semantic Analysis in a small corpus of text". <i>Consciousness and Cognition</i>. <b>56</b>: <span class="nowrap">178–</span>187. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1610.01520">1610.01520</a></span>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.concog.2017.09.004">10.1016/j.concog.2017.09.004</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/28943127">28943127</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:195347873">195347873</a>.</cite></span>

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2></div>

<ul><li><a rel="nofollow" class="external text" href="https://wikipedia2vec.github.io/demo/">Wikipedia2Vec</a><a rel="nofollow" class="external autonumber" href="http://arxiv.org/abs/1812.06280">[1]</a> (<a rel="nofollow" class="external text" href="https://wikipedia2vec.github.io/wikipedia2vec/">introduction</a>)</li></ul>

<div class="mw-heading mw-heading3"><h3 id="Implementations">Implementations</h3></div>

<li><a rel="nofollow" class="external text" href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.feature.Word2Vec.html">Python (Spark)</a></li>

<li><a rel="nofollow" class="external text" href="https://github.com/tensorflow/tensorflow/blob/r1.1/tensorflow/examples/tutorials/word2vec/word2vec_basic.py">Python (TensorFlow)</a></li>

<li><a rel="nofollow" class="external text" href="http://radimrehurek.com/gensim/models/word2vec.html">Python (Gensim)</a></li>

<li><a rel="nofollow" class="external text" href="https://github.com/deeplearning4j/deeplearning4j">Java/Scala</a></li>

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></div><div role="navigation" class="navbox" aria-labelledby="Natural_language_processing454" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Natural_language_processing454" style="font-size:114%;margin:0 4em"><a href="Natural_language_processing" title="Natural language processing">Natural language processing</a></div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%">General terms</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AI-complete" title="AI-complete">AI-complete</a></li>

<li><a href="Bag-of-words_model" title="Bag-of-words model">Bag-of-words</a></li>

<ul><li><a href="Bigram" title="Bigram">Bigram</a></li>

<li><a href="Trigram" title="Trigram">Trigram</a></li></ul></li>

<li><a href="Computational_linguistics" title="Computational linguistics">Computational linguistics</a></li>

<li><a href="Natural_language_understanding" title="Natural language understanding">Natural language understanding</a></li>

<li><a href="Stop_word" title="Stop word">Stop words</a></li>

<li><a href="Text_processing" title="Text processing">Text processing</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Text_mining" title="Text mining">Text analysis</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Argument_mining" title="Argument mining">Argument mining</a></li>

<li><a href="Collocation_extraction" title="Collocation extraction">Collocation extraction</a></li>

<li><a href="Concept_mining" title="Concept mining">Concept mining</a></li>

<li><a href="Coreference#Coreference_resolution" title="Coreference">Coreference resolution</a></li>

<li><a href="Deep_linguistic_processing" title="Deep linguistic processing">Deep linguistic processing</a></li>

<li><a href="Distant_reading" title="Distant reading">Distant reading</a></li>

<li><a href="Information_extraction" title="Information extraction">Information extraction</a></li>

<li><a href="Named-entity_recognition" title="Named-entity recognition">Named-entity recognition</a></li>

<li><a href="Ontology_learning" title="Ontology learning">Ontology learning</a></li>

<li><a href="Parsing" title="Parsing">Parsing</a>

<ul><li><a href="Semantic_parsing" title="Semantic parsing">Semantic parsing</a></li>

<li><a href="Syntactic_parsing_(computational_linguistics)" title="Syntactic parsing (computational linguistics)">Syntactic parsing</a></li></ul></li>

<li><a href="Part-of-speech_tagging" title="Part-of-speech tagging">Part-of-speech tagging</a></li>

<li><a href="Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li>

<li><a href="Semantic_role_labeling" title="Semantic role labeling">Semantic role labeling</a></li>

<li><a href="Semantic_decomposition_(natural_language_processing)" title="Semantic decomposition (natural language processing)">Semantic decomposition</a></li>

<li><a href="Semantic_similarity" title="Semantic similarity">Semantic similarity</a></li>

<li><a href="Sentiment_analysis" title="Sentiment analysis">Sentiment analysis</a></li></ul>

<ul><li><a href="Terminology_extraction" title="Terminology extraction">Terminology extraction</a></li>

<li><a href="Text_mining" title="Text mining">Text mining</a></li>

<li><a href="Textual_entailment" title="Textual entailment">Textual entailment</a></li>

<li><a href="Truecasing" title="Truecasing">Truecasing</a></li>

<li><a href="Word-sense_disambiguation" title="Word-sense disambiguation">Word-sense disambiguation</a></li>

<li><a href="Word-sense_induction" title="Word-sense induction">Word-sense induction</a></li></ul>

</div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th id="Text_segmentation21" scope="row" class="navbox-group" style="width:1%"><a href="Text_segmentation" title="Text segmentation">Text segmentation</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Compound-term_processing" title="Compound-term processing">Compound-term processing</a></li>

<li><a href="Lemmatisation" class="mw-redirect" title="Lemmatisation">Lemmatisation</a></li>

<li><a href="Lexical_analysis" title="Lexical analysis">Lexical analysis</a></li>

<li><a href="Shallow_parsing" title="Shallow parsing">Text chunking</a></li>

<li><a href="Stemming" title="Stemming">Stemming</a></li>

<li><a href="Sentence_boundary_disambiguation" title="Sentence boundary disambiguation">Sentence segmentation</a></li>

<li><a href="Word#Word_boundaries" title="Word">Word segmentation</a></li></ul>

</div></td></tr></tbody></table><div>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Automatic_summarization" title="Automatic summarization">Automatic summarization</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Multi-document_summarization" title="Multi-document summarization">Multi-document summarization</a></li>

<li><a href="Sentence_extraction" title="Sentence extraction">Sentence extraction</a></li>

<li><a href="Text_simplification" title="Text simplification">Text simplification</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Machine_translation" title="Machine translation">Machine translation</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Computer-assisted_translation" title="Computer-assisted translation">Computer-assisted</a></li>

<li><a href="Example-based_machine_translation" title="Example-based machine translation">Example-based</a></li>

<li><a href="Rule-based_machine_translation" title="Rule-based machine translation">Rule-based</a></li>

<li><a href="Statistical_machine_translation" title="Statistical machine translation">Statistical</a></li>

<li><a href="Transfer-based_machine_translation" title="Transfer-based machine translation">Transfer-based</a></li>

<li><a href="Neural_machine_translation" title="Neural machine translation">Neural</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Distributional_semantics" title="Distributional semantics">Distributional semantics</a> models</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<li><a href="Document-term_matrix" title="Document-term matrix">Document-term matrix</a></li>

<li><a href="Explicit_semantic_analysis" title="Explicit semantic analysis">Explicit semantic analysis</a></li>

<li><a href="FastText" title="FastText">fastText</a></li>

<li><a href="GloVe" title="GloVe">GloVe</a></li>

<li><a href="Language_model" title="Language model">Language model</a> (<a href="Large_language_model" title="Large language model">large</a>)</li>

<li><a href="Latent_semantic_analysis" title="Latent semantic analysis">Latent semantic analysis</a></li>

<li><a href="Word_embedding" title="Word embedding">Word embedding</a></li>

</ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Language_resource" title="Language resource">Language resources</a>,<br>datasets and corpora</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Types and<br>standards</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Corpus_linguistics" title="Corpus linguistics">Corpus linguistics</a></li>

<li><a href="Lexical_resource" title="Lexical resource">Lexical resource</a></li>

<li><a href="Linguistic_Linked_Open_Data" title="Linguistic Linked Open Data">Linguistic Linked Open Data</a></li>

<li><a href="Machine-readable_dictionary" title="Machine-readable dictionary">Machine-readable dictionary</a></li>

<li><a href="Parallel_text" title="Parallel text">Parallel text</a></li>

<li><a href="PropBank" title="PropBank">PropBank</a></li>

<li><a href="Semantic_network" title="Semantic network">Semantic network</a></li>

<li><a href="Simple_Knowledge_Organization_System" title="Simple Knowledge Organization System">Simple Knowledge Organization System</a></li>

<li><a href="Speech_corpus" title="Speech corpus">Speech corpus</a></li>

<li><a href="Text_corpus" title="Text corpus">Text corpus</a></li>

<li><a href="Thesaurus_(information_retrieval)" title="Thesaurus (information retrieval)">Thesaurus (information retrieval)</a></li>

<li><a href="Treebank" title="Treebank">Treebank</a></li>

<li><a href="Universal_Dependencies" title="Universal Dependencies">Universal Dependencies</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Data</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="BabelNet" title="BabelNet">BabelNet</a></li>

<li><a href="Bank_of_English" title="Bank of English">Bank of English</a></li>

<li><a href="DBpedia" title="DBpedia">DBpedia</a></li>

<li><a href="FrameNet" title="FrameNet">FrameNet</a></li>

<li><a href="Google_Ngram_Viewer" class="mw-redirect" title="Google Ngram Viewer">Google Ngram Viewer</a></li>

<li><a href="WordNet" title="WordNet">WordNet</a></li>

<li><a href="Wikidata" title="Wikidata">Wikidata</a></li></ul>

</div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Automatic_identification_and_data_capture" title="Automatic identification and data capture">Automatic identification<br>and data capture</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Speech_recognition" title="Speech recognition">Speech recognition</a></li>

<li><a href="Speech_segmentation" title="Speech segmentation">Speech segmentation</a></li>

<li><a href="Speech_synthesis" title="Speech synthesis">Speech synthesis</a></li>

<li><a href="Natural_language_generation" title="Natural language generation">Natural language generation</a></li>

<li><a href="Optical_character_recognition" title="Optical character recognition">Optical character recognition</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Topic_model" title="Topic model">Topic model</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Document_classification" title="Document classification">Document classification</a></li>

<li><a href="Latent_Dirichlet_allocation" title="Latent Dirichlet allocation">Latent Dirichlet allocation</a></li>

<li><a href="Pachinko_allocation" title="Pachinko allocation">Pachinko allocation</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Computer-assisted_reviewing" title="Computer-assisted reviewing">Computer-assisted<br>reviewing</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Automated_essay_scoring" title="Automated essay scoring">Automated essay scoring</a></li>

<li><a href="Concordancer" title="Concordancer">Concordancer</a></li>

<li><a href="Grammar_checker" title="Grammar checker">Grammar checker</a></li>

<li><a href="Predictive_text" title="Predictive text">Predictive text</a></li>

<li><a href="Pronunciation_assessment" title="Pronunciation assessment">Pronunciation assessment</a></li>

<li><a href="Spell_checker" title="Spell checker">Spell checker</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="Natural-language_user_interface" title="Natural-language user interface">Natural language<br>user interface</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Chatbot" title="Chatbot">Chatbot</a></li>

<li><a href="Interactive_fiction" title="Interactive fiction">Interactive fiction</a></li>

<li><a href="Question_answering" title="Question answering">Question answering</a></li>

<li><a href="Virtual_assistant" title="Virtual assistant">Virtual assistant</a></li>

<li><a href="Voice_user_interface" title="Voice user interface">Voice user interface</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Related</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Formal_semantics_(natural_language)" title="Formal semantics (natural language)">Formal semantics</a></li>

<li><a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li>

<li><a href="Natural_Language_Toolkit" title="Natural Language Toolkit">Natural Language Toolkit</a></li>

<li><a href="SpaCy" title="SpaCy">spaCy</a></li></ul>

</div></td></tr></tbody></table></div>

<div class="navbox-styles"></div><div role="navigation" class="navbox" aria-labelledby="Artificial_intelligence_(AI)426" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Artificial_intelligence_(AI)426" style="font-size:114%;margin:0 4em"><a href="Artificial_intelligence" title="Artificial intelligence">Artificial intelligence</a> (AI)</div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li><a href="History_of_artificial_intelligence" title="History of artificial intelligence">History</a>

<ul><li><a href="Timeline_of_artificial_intelligence" title="Timeline of artificial intelligence">timeline</a></li></ul></li>

<li><a href="List_of_artificial_intelligence_companies" title="List of artificial intelligence companies">Companies</a></li>

<li><a href="List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Parameter" title="Parameter">Parameter</a>

<ul><li><a href="Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">Hyperparameter</a></li></ul></li>

<li><a href="Loss_functions_for_classification" title="Loss functions for classification">Loss functions</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a>

<ul><li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Double_descent" title="Double descent">Double descent</a></li>

<li><a href="Overfitting" title="Overfitting">Overfitting</a></li></ul></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Gradient_descent" title="Gradient descent">Gradient descent</a>

<li><a href="Quasi-Newton_method" title="Quasi-Newton method">Quasi-Newton method</a></li>

<li><a href="Conjugate_gradient_method" title="Conjugate gradient method">Conjugate gradient method</a></li></ul></li>

<li><a href="Backpropagation" title="Backpropagation">Backpropagation</a></li>

<li><a href="Attention_(machine_learning)" title="Attention (machine learning)">Attention</a></li>

<li><a href="Convolution" title="Convolution">Convolution</a></li>

<li><a href="Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization</a>

<ul><li><a href="Batch_normalization" title="Batch normalization">Batchnorm</a></li></ul></li>

<li><a href="Activation_function" title="Activation function">Activation</a>

<ul><li><a href="Softmax_function" title="Softmax function">Softmax</a></li>

<li><a href="Sigmoid_function" title="Sigmoid function">Sigmoid</a></li>

<li><a href="Rectifier_(neural_networks)" title="Rectifier (neural networks)">Rectifier</a></li></ul></li>

<li><a href="Gating_mechanism" title="Gating mechanism">Gating</a></li>

<li><a href="Weight_initialization" title="Weight initialization">Weight initialization</a></li>

<li><a href="Regularization_(mathematics)" title="Regularization (mathematics)">Regularization</a></li>

<li><a href="Training%2C_validation%2C_and_test_data_sets" title="Training, validation, and test data sets">Datasets</a>

<ul><li><a href="Data_augmentation" title="Data augmentation">Augmentation</a></li></ul></li>

<li><a href="Prompt_engineering" title="Prompt engineering">Prompt engineering</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a>

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Imitation_learning" title="Imitation learning">Imitation</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li></ul></li>

<li><a href="Diffusion_process" title="Diffusion process">Diffusion</a></li>

<li><a href="Latent_diffusion_model" title="Latent diffusion model">Latent diffusion model</a></li>

<li><a href="Autoregressive_model" title="Autoregressive model">Autoregression</a></li>

<li><a href="Adversarial_machine_learning" title="Adversarial machine learning">Adversary</a></li>

<li><a href="Uncanny_valley" title="Uncanny valley">Uncanny valley</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reflection_(artificial_intelligence)" class="mw-redirect" title="Reflection (artificial intelligence)">Reflection</a></li>

<li><a href="Recursive_self-improvement" title="Recursive self-improvement">Recursive self-improvement</a></li>

<li><a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li>

<li><a href="Word_embedding" title="Word embedding">Word embedding</a></li>

<li><a href="Vibe_coding" title="Vibe coding">Vibe coding</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Applications</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Machine_learning" title="Machine learning">Machine learning</a>

<ul><li><a href="Prompt_engineering#In-context_learning" title="Prompt engineering">In-context learning</a></li></ul></li>

<li><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Artificial neural network</a>

<ul><li><a href="Deep_learning" title="Deep learning">Deep learning</a></li></ul></li>

<li><a href="Language_model" title="Language model">Language model</a>

<ul><li><a href="Large_language_model" title="Large language model">Large language model</a></li>

<li><a href="Reasoning_language_model" title="Reasoning language model">Reasoning language model</a></li>

<li><a href="Model_Context_Protocol" title="Model Context Protocol">Model Context Protocol</a></li>

<li><a href="Intelligent_agent" title="Intelligent agent">Intelligent agent</a></li>

<li><a href="Artificial_human_companion" title="Artificial human companion">Artificial human companion</a></li>

<li><a href="Humanity's_Last_Exam" title="Humanity's Last Exam">Humanity's Last Exam</a></li>

<li><a href="Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence (AGI)</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Implementations</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Audio–visual</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="WaveNet" title="WaveNet">WaveNet</a></li>

<li><a href="Human_image_synthesis" title="Human image synthesis">Human image synthesis</a></li>

<li><a href="Computer_vision" title="Computer vision">Computer vision</a></li>

<li><a href="Deep_learning_speech_synthesis" title="Deep learning speech synthesis">Speech synthesis</a>

<li><a href="ElevenLabs" title="ElevenLabs">ElevenLabs</a></li></ul></li>

<li><a href="Speech_recognition" title="Speech recognition">Speech recognition</a>

<ul><li><a href="Whisper_(speech_recognition_system)" title="Whisper (speech recognition system)">Whisper</a></li></ul></li>

<li><a href="Facial_recognition_system" title="Facial recognition system">Facial recognition</a></li>

<li><a href="AlphaFold" title="AlphaFold">AlphaFold</a></li>

<li><a href="Text-to-image_model" title="Text-to-image model">Text-to-image models</a>

<ul><li><a href="Aurora_(text-to-image_model)" class="mw-redirect" title="Aurora (text-to-image model)">Aurora</a></li>

<li><a href="Adobe_Firefly" title="Adobe Firefly">Firefly</a></li>

<li><a href="Ideogram_(text-to-image_model)" title="Ideogram (text-to-image model)">Ideogram</a></li>

<li><a href="Imagen_(text-to-image_model)" title="Imagen (text-to-image model)">Imagen</a></li>

<li><a href="Midjourney" title="Midjourney">Midjourney</a></li>

<li><a href="Recraft" title="Recraft">Recraft</a></li>

<li><a href="Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a></li></ul></li>

<li><a href="Text-to-video_model" title="Text-to-video model">Text-to-video models</a>

<ul><li><a href="Dream_Machine_(text-to-video_model)" title="Dream Machine (text-to-video model)">Dream Machine</a></li>

<li><a href="Runway_(company)#Services_and_technologies" title="Runway (company)">Runway Gen</a></li>

<li><a href="MiniMax_(company)#Hailuo_AI" title="MiniMax (company)">Hailuo AI</a></li>

<li><a href="Kling_(text-to-video_model)" class="mw-redirect" title="Kling (text-to-video model)">Kling</a></li>

<li><a href="Music_and_artificial_intelligence" title="Music and artificial intelligence">Music generation</a>

<ul><li><a href="Riffusion" title="Riffusion">Riffusion</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Text</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul>

<li><a href="GloVe" title="GloVe">GloVe</a></li>

<li><a href="Llama_(language_model)" title="Llama (language model)">Llama</a></li>

<li><a href="Chinchilla_(language_model)" title="Chinchilla (language model)">Chinchilla AI</a></li>

<li><a href="ChatGPT" title="ChatGPT">ChatGPT</a></li>

<li><a href="Claude_(language_model)" title="Claude (language model)">Claude</a></li>

<li><a href="Gemini_(language_model)" title="Gemini (language model)">Gemini</a>

<ul><li><a href="Gemini_(chatbot)" title="Gemini (chatbot)">chatbot</a></li></ul></li>

<li><a href="LaMDA" title="LaMDA">LaMDA</a></li>

<li><a href="BLOOM_(language_model)" title="BLOOM (language model)">BLOOM</a></li>

<li><a href="Project_Debater" title="Project Debater">Project Debater</a></li>

<li><a href="IBM_Watson" title="IBM Watson">IBM Watson</a></li>

<li><a href="IBM_Watsonx" title="IBM Watsonx">IBM Watsonx</a></li>

<li><a href="IBM_Granite" title="IBM Granite">Granite</a></li>

<li><a href="Huawei_PanGu" title="Huawei PanGu">PanGu-Σ</a></li>

<li><a href="DeepSeek_(chatbot)" title="DeepSeek (chatbot)">DeepSeek</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Decisional</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlphaGo" title="AlphaGo">AlphaGo</a></li>

<li><a href="AlphaZero" title="AlphaZero">AlphaZero</a></li>

<li><a href="OpenAI_Five" title="OpenAI Five">OpenAI Five</a></li>

<li><a href="Self-driving_car" title="Self-driving car">Self-driving car</a></li>

<li><a href="MuZero" title="MuZero">MuZero</a></li>

<li><a href="Action_selection" title="Action selection">Action selection</a>

<ul><li><a href="AutoGPT" title="AutoGPT">AutoGPT</a></li></ul></li>

<li><a href="Robot_control" title="Robot control">Robot control</a></li></ul>

</div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Alan_Turing" title="Alan Turing">Alan Turing</a></li>

<li><a href="Warren_Sturgis_McCulloch" title="Warren Sturgis McCulloch">Warren Sturgis McCulloch</a></li>

<li><a href="Walter_Pitts" title="Walter Pitts">Walter Pitts</a></li>

<li><a href="John_von_Neumann" title="John von Neumann">John von Neumann</a></li>

<li><a href="Claude_Shannon" title="Claude Shannon">Claude Shannon</a></li>

<li><a href="Shun'ichi_Amari" title="Shun'ichi Amari">Shun'ichi Amari</a></li>

<li><a href="Kunihiko_Fukushima" title="Kunihiko Fukushima">Kunihiko Fukushima</a></li>

<li><a href="Takeo_Kanade" title="Takeo Kanade">Takeo Kanade</a></li>

<li><a href="Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a></li>

<li><a href="John_McCarthy_(computer_scientist)" title="John McCarthy (computer scientist)">John McCarthy</a></li>

<li><a href="Nathaniel_Rochester_(computer_scientist)" title="Nathaniel Rochester (computer scientist)">Nathaniel Rochester</a></li>

<li><a href="Allen_Newell" title="Allen Newell">Allen Newell</a></li>

<li><a href="Cliff_Shaw" title="Cliff Shaw">Cliff Shaw</a></li>

<li><a href="Herbert_A._Simon" title="Herbert A. Simon">Herbert A. Simon</a></li>

<li><a href="Oliver_Selfridge" title="Oliver Selfridge">Oliver Selfridge</a></li>

<li><a href="Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a></li>

<li><a href="Bernard_Widrow" title="Bernard Widrow">Bernard Widrow</a></li>

<li><a href="Joseph_Weizenbaum" title="Joseph Weizenbaum">Joseph Weizenbaum</a></li>

<li><a href="Seymour_Papert" title="Seymour Papert">Seymour Papert</a></li>

<li><a href="Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a></li>

<li><a href="Paul_Werbos" title="Paul Werbos">Paul Werbos</a></li>

<li><a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li>

<li><a href="John_Hopfield" title="John Hopfield">John Hopfield</a></li>

<li><a href="J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a></li>

<li><a href="Yann_LeCun" title="Yann LeCun">Yann LeCun</a></li>

<li><a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li>

<li><a href="Lotfi_A._Zadeh" title="Lotfi A. Zadeh">Lotfi A. Zadeh</a></li>

<li><a href="Stephen_Grossberg" title="Stephen Grossberg">Stephen Grossberg</a></li>

<li><a href="Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Alex Graves</a></li>

<li><a href="James_Goodnight" title="James Goodnight">James Goodnight</a></li>

<li><a href="Andrew_Ng" title="Andrew Ng">Andrew Ng</a></li>

<li><a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li>

<li><a href="Alex_Krizhevsky" title="Alex Krizhevsky">Alex Krizhevsky</a></li>

<li><a href="Ian_Goodfellow" title="Ian Goodfellow">Ian Goodfellow</a></li>

<li><a href="Demis_Hassabis" title="Demis Hassabis">Demis Hassabis</a></li>

<li><a href="David_Silver_(computer_scientist)" title="David Silver (computer scientist)">David Silver</a></li>

<li><a href="Andrej_Karpathy" title="Andrej Karpathy">Andrej Karpathy</a></li>

<li><a href="Ashish_Vaswani" title="Ashish Vaswani">Ashish Vaswani</a></li>

<li><a href="Noam_Shazeer" title="Noam Shazeer">Noam Shazeer</a></li>

<li><a href="Aidan_Gomez" title="Aidan Gomez">Aidan Gomez</a></li>

<li><a href="Mustafa_Suleyman" title="Mustafa Suleyman">Mustafa Suleyman</a></li>

<li><a href="Fran%C3%A7ois_Chollet" title="François Chollet">François Chollet</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Architectures</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a></li>

<li><a href="Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision transformer (ViT)</a></li></ul></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network (RNN)</a></li>

<li><a href="Long_short-term_memory" title="Long short-term memory">Long short-term memory (LSTM)</a></li>

<li><a href="Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit (GRU)</a></li>

<li><a href="Echo_state_network" title="Echo state network">Echo state network</a></li>

<li><a href="Multilayer_perceptron" title="Multilayer perceptron">Multilayer perceptron (MLP)</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network (CNN)</a></li>

<li><a href="Residual_neural_network" title="Residual neural network">Residual neural network (RNN)</a></li>

<li><a href="Highway_network" title="Highway network">Highway network</a></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Variational_autoencoder" title="Variational autoencoder">Variational autoencoder (VAE)</a></li>

<li><a href="Generative_adversarial_network" title="Generative adversarial network">Generative adversarial network (GAN)</a></li>

<li><a href="Graph_neural_network" title="Graph neural network">Graph neural network (GNN)</a></li></ul>

</div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li><span class="noviewer" typeof="mw:File"><span title="Category"></span></span> Category</li></ul>

</div></td></tr></tbody></table></div>

</p></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-08-02" href="https://en.wikipedia.org/wiki/?title=Word2vec&oldid=1303914540">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>